

<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>cortex.built_ins.datasets.toysets &mdash; Cortex2.0  documentation</title>
  

  
  
  
  

  

  
  
    

  

  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
    <link rel="index" title="Index" href="../../../../genindex.html" />
    <link rel="search" title="Search" href="../../../../search.html" /> 

  
  <script src="../../../../_static/js/modernizr.min.js"></script>

</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">

    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search">
          

          
            <a href="../../../../index.html" class="icon icon-home"> Cortex2.0
          

          
          </a>

          
            
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">User Documentation</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../install.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../getting_started.html">Getting Started</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../modules.html">cortex</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../develop.html">Develop</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../build.html">Custom demos</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../build.html#a-walkthrough-a-custom-classifier">A walkthrough a custom classifier:</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../build.html#defining-losses-and-results">Defining losses and results</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../build.html#visualization">Visualization</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../build.html#putting-it-together">Putting it together</a></li>
</ul>

            
          
        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../../../../index.html">Cortex2.0</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
        
          <li><a href="../../../index.html">Module code</a> &raquo;</li>
        
      <li>cortex.built_ins.datasets.toysets</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <h1>Source code for cortex.built_ins.datasets.toysets</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">:mod:`cortex2.lib.toysets` -- Small toy datasets for interpretable</span>
<span class="sd">      experimentation</span>
<span class="sd">==================================================================================</span>

<span class="sd">.. module:: toysets</span>
<span class="sd">   :platform: Unix</span>
<span class="sd">   :synopsis: Basically everything found in this website</span>
<span class="sd">      &lt;https://cs.joensuu.fi/sipu/datasets/&gt;_ (for now).</span>

<span class="sd">Collection of datasets (mostly 2D) used primarily for benchmarking of</span>
<span class="sd">inference algorithms and interpretable experiments in the input space.</span>

<span class="sd">TODOs</span>
<span class="sd">-----</span>
<span class="sd">1. Include common datasets for toying with GANs, like the balanced 2-moons</span>
<span class="sd">2. Fix module title once a proper packaging scheme is introduced</span>

<span class="sd">&quot;&quot;&quot;</span>

<span class="kn">import</span> <span class="nn">errno</span>
<span class="kn">import</span> <span class="nn">itertools</span> <span class="k">as</span> <span class="nn">it</span>
<span class="kn">import</span> <span class="nn">os</span>

<span class="kn">import</span> <span class="nn">torch</span>
<span class="kn">import</span> <span class="nn">torch.utils.data</span> <span class="k">as</span> <span class="nn">data</span>


<span class="n">__author__</span> <span class="o">=</span> <span class="s1">&#39;Tsirigotis Christos&#39;</span>
<span class="n">__author_email__</span> <span class="o">=</span> <span class="s1">&#39;tsirif@gmail.com&#39;</span>

<span class="n">DATASETS</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;G2&quot;</span><span class="p">,</span> <span class="s2">&quot;S_set&quot;</span><span class="p">,</span> <span class="s2">&quot;A_set&quot;</span><span class="p">,</span> <span class="s2">&quot;DIM_set&quot;</span><span class="p">,</span> <span class="s2">&quot;Unbalance&quot;</span><span class="p">,</span>
            <span class="s2">&quot;Aggregation&quot;</span><span class="p">,</span> <span class="s2">&quot;Compound&quot;</span><span class="p">,</span> <span class="s2">&quot;Pathbased&quot;</span><span class="p">,</span> <span class="s2">&quot;Spiral&quot;</span><span class="p">,</span>
            <span class="s2">&quot;D31&quot;</span><span class="p">,</span> <span class="s2">&quot;R15&quot;</span><span class="p">,</span> <span class="s2">&quot;Jain&quot;</span><span class="p">,</span> <span class="s2">&quot;Flame&quot;</span><span class="p">]</span>
<span class="n">DIM_VARIANT_DATASETS</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;G2&quot;</span><span class="p">,</span> <span class="s2">&quot;DIM_set&quot;</span><span class="p">]</span>
<span class="n">SD_VARIANT_DATASETS</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;G2&quot;</span><span class="p">]</span>
<span class="n">NUM_VARIANT_DATASETS</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;S_set&quot;</span><span class="p">,</span> <span class="s2">&quot;A_set&quot;</span><span class="p">]</span>


<div class="viewcode-block" id="make_tds_random_and_split"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.make_tds_random_and_split">[docs]</a><span class="k">def</span> <span class="nf">make_tds_random_and_split</span><span class="p">(</span><span class="n">C</span><span class="p">):</span>
    <span class="sd">&#39;&#39;&#39;Wraps Toyset class to add random splitting.</span>

<span class="sd">    Args:</span>
<span class="sd">        C: Toyset data class to be wrapped</span>

<span class="sd">    Returns:</span>
<span class="sd">        RandomSplitting class that wraps Toyset data class</span>

<span class="sd">    &#39;&#39;&#39;</span>
    <span class="k">class</span> <span class="nc">RandomSplitting</span><span class="p">(</span><span class="n">C</span><span class="p">):</span>
        <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="n">idx</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">split</span><span class="o">=.</span><span class="mi">8</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
            <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">idx</span> <span class="o">=</span> <span class="n">idx</span> <span class="k">if</span> <span class="n">idx</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">torch</span><span class="o">.</span><span class="n">randperm</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span>
            <span class="n">tensors_</span> <span class="o">=</span> <span class="p">[]</span>

            <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tensors</span><span class="p">)):</span>
                <span class="k">if</span> <span class="n">split</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
                    <span class="n">tensors_</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
                        <span class="bp">self</span><span class="o">.</span><span class="n">tensors</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="bp">self</span><span class="o">.</span><span class="n">idx</span><span class="p">][:</span><span class="nb">int</span><span class="p">(</span><span class="n">split</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="p">))])</span>
                <span class="k">else</span><span class="p">:</span>
                    <span class="n">tensors_</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
                        <span class="bp">self</span><span class="o">.</span><span class="n">tensors</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="bp">self</span><span class="o">.</span><span class="n">idx</span><span class="p">][</span><span class="nb">int</span><span class="p">(</span><span class="n">split</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span> <span class="o">-</span> <span class="mi">1</span><span class="p">:])</span>

            <span class="bp">self</span><span class="o">.</span><span class="n">tensors</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">tensors_</span><span class="p">)</span>

    <span class="k">return</span> <span class="n">RandomSplitting</span></div>


<span class="k">class</span> <span class="nc">_SmallDataset</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">TensorDataset</span><span class="p">):</span>

    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">root</span><span class="p">,</span> <span class="o">*</span><span class="n">select</span><span class="p">,</span>
                 <span class="n">stardardize</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">load</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">download</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Download or load a small dataset.</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        root : str</span>
<span class="sd">           Names the path to a directory in which the dataset is or will be</span>
<span class="sd">           located</span>
<span class="sd">        select : positional arguments</span>
<span class="sd">           Some extra, possibly unnecessary arguments, to specify a particular</span>
<span class="sd">           dataset from a family of datasets to be loaded.</span>

<span class="sd">           What it can be, depends on a particular dataset, **please refer** to</span>
<span class="sd">           its documentation.</span>
<span class="sd">        stardardize : bool, optional, default=False</span>
<span class="sd">           If True, perform small preprocessing of the datasets:</span>
<span class="sd">           ``(X - mean)/std``, when preparing the dataset for loading</span>
<span class="sd">        load : bool, optional, default=False</span>
<span class="sd">           If True, load the dataset from local directory `root`</span>
<span class="sd">        download : bool, optional, default=False</span>
<span class="sd">           If True, download the dataset to local directory `root`</span>
<span class="sd">           from an online source</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">root</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">expanduser</span><span class="p">(</span><span class="n">root</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">stardardize</span> <span class="o">=</span> <span class="n">stardardize</span>

        <span class="k">if</span> <span class="n">download</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">download</span><span class="p">()</span>

        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">check_exists</span><span class="p">():</span>
            <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s1">&#39;Dataset not found.&#39;</span> <span class="o">+</span>
                               <span class="s1">&#39; You can use download=True to download it&#39;</span><span class="p">)</span>

        <span class="k">if</span> <span class="n">load</span><span class="p">:</span>
            <span class="n">data</span><span class="p">,</span> <span class="n">target</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">prepare</span><span class="p">(</span><span class="o">*</span><span class="n">select</span><span class="p">)</span>
            <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">download</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Download, and unzip in the correct location.</span>
<span class="sd">        Returns:</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="kn">import</span> <span class="nn">urllib</span>
        <span class="kn">import</span> <span class="nn">zipfile</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">check_exists</span><span class="p">():</span>
            <span class="k">return</span>

        <span class="c1"># download files</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">root</span><span class="p">)</span>
        <span class="k">except</span> <span class="ne">OSError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
            <span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">errno</span> <span class="o">==</span> <span class="n">errno</span><span class="o">.</span><span class="n">EEXIST</span><span class="p">:</span>
                <span class="k">pass</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="k">raise</span>

        <span class="k">for</span> <span class="n">url</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">urls</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Downloading &#39;</span> <span class="o">+</span> <span class="n">url</span><span class="p">)</span>
            <span class="n">data</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">request</span><span class="o">.</span><span class="n">urlopen</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
            <span class="n">filename</span> <span class="o">=</span> <span class="n">url</span><span class="o">.</span><span class="n">rpartition</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)[</span><span class="mi">2</span><span class="p">]</span>
            <span class="n">file_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">root</span><span class="p">,</span> <span class="n">filename</span><span class="p">)</span>
            <span class="n">ext</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">splitext</span><span class="p">(</span><span class="n">file_path</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span>
            <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">file_path</span><span class="p">,</span> <span class="s1">&#39;wb&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
                <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">read</span><span class="p">())</span>
            <span class="k">if</span> <span class="n">ext</span> <span class="o">==</span> <span class="s1">&#39;.zip&#39;</span><span class="p">:</span>
                <span class="k">with</span> <span class="n">zipfile</span><span class="o">.</span><span class="n">ZipFile</span><span class="p">(</span><span class="n">file_path</span><span class="p">)</span> <span class="k">as</span> <span class="n">zip_f</span><span class="p">:</span>
                    <span class="n">zip_f</span><span class="o">.</span><span class="n">extractall</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">root</span><span class="p">)</span>
                <span class="n">os</span><span class="o">.</span><span class="n">unlink</span><span class="p">(</span><span class="n">file_path</span><span class="p">)</span>

        <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Done!&#39;</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">prepare</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">select</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>

<span class="sd">        Args:</span>
<span class="sd">            *select:</span>

<span class="sd">        Returns:</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">datafile</span><span class="p">,</span> <span class="n">labelfile</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">(</span><span class="o">*</span><span class="n">select</span><span class="p">)</span>
        <span class="n">data_filepath</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">root</span><span class="p">,</span> <span class="n">datafile</span><span class="p">)</span>
        <span class="n">label_filepath</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">root</span><span class="p">,</span> <span class="n">labelfile</span><span class="p">)</span>
        <span class="n">data</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="n">target</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">data_filepath</span><span class="p">)</span> <span class="k">as</span> <span class="n">data_f</span><span class="p">,</span> <span class="nb">open</span><span class="p">(</span><span class="n">label_filepath</span><span class="p">)</span> <span class="k">as</span> <span class="n">label_f</span><span class="p">:</span>
            <span class="k">for</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">data_f</span><span class="p">,</span> <span class="n">it</span><span class="o">.</span><span class="n">islice</span><span class="p">(</span><span class="n">label_f</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sync_files</span><span class="p">,</span> <span class="kc">None</span><span class="p">)):</span>
                <span class="n">data</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="nb">int</span><span class="p">,</span> <span class="n">x</span><span class="o">.</span><span class="n">split</span><span class="p">())))</span>
                <span class="n">target</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">y</span><span class="p">))</span>
        <span class="n">data</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
        <span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">(</span><span class="n">target</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">stardardize</span><span class="p">:</span>
            <span class="n">data_mean</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">keepdim</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
            <span class="n">data_std</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">std</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">keepdim</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
            <span class="n">data</span> <span class="o">=</span> <span class="p">(</span><span class="n">data</span> <span class="o">-</span> <span class="n">data_mean</span><span class="p">)</span> <span class="o">/</span> <span class="n">data_std</span>

        <span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">target</span>

    <span class="k">def</span> <span class="nf">files</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">select</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>

<span class="sd">        Args:</span>
<span class="sd">            *select:</span>

<span class="sd">        Returns:</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="s1">&#39;&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span>

    <span class="k">def</span> <span class="nf">check_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>

<span class="sd">        Returns:</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="kc">True</span>


<div class="viewcode-block" id="G2"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.G2">[docs]</a><span class="k">class</span> <span class="nc">G2</span><span class="p">(</span><span class="n">_SmallDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Download and use G2 dataset.</span>

<span class="sd">    Select arguments</span>
<span class="sd">    ----------------</span>
<span class="sd">    dim : int</span>
<span class="sd">       Dimension of the input space</span>
<span class="sd">    sd : int</span>
<span class="sd">       Standard deviation of the Gaussian used to generate the 2 modes</span>

<span class="sd">    See possible values below.</span>

<span class="sd">    -----------------------------------------------------------------------</span>
<span class="sd">    G2 datasets creation</span>
<span class="sd">    -----------------------------------------------------------------------</span>

<span class="sd">    The datasets include two Gaussian normal distributions:</span>

<span class="sd">    Dataset name:    G2-dim-sd</span>
<span class="sd">    Centroid 1:      [500,500, ...]</span>
<span class="sd">    Centroid 2:      [600,600, ...]</span>
<span class="sd">    Dimensions:      dim = 1,2,4,8,16, ... 1024</span>
<span class="sd">    St.Dev:          sd  = 10,20,30,40 ... 100</span>

<span class="sd">    They have been created using the following C-language code:</span>

<span class="sd">    Calculate random value in (0,1]:</span>

<span class="sd">    U = (double)(rand()+1)/(double)(RAND_MAX+1);</span>
<span class="sd">    V = (double)(rand()+1)/(double)(RAND_MAX+1);</span>

<span class="sd">    Box-Muller method to create two independent standard</span>
<span class="sd">    one-dimensional Gaussian samples:</span>

<span class="sd">    X = sqrt(-2*log(U))*cos(2*3.14159*V);  /* pi = 3.14159 */</span>
<span class="sd">    Y = sqrt(-2*log(U))*sin(2*3.14159*V);</span>

<span class="sd">    Adjust mean and deviation:</span>

<span class="sd">    X_final = 500 + s * X;    /* mean + deviation * X */</span>
<span class="sd">    Y_final = 600 + s * Y;</span>

<span class="sd">    The points are stored in the files so that:</span>
<span class="sd">    - First 1024 points are from the cluster 1</span>
<span class="sd">    - Rest  1024 points are from the cluster 2</span>

<span class="sd">    -----------------------------------------------------------------------</span>

<span class="sd">    P. Fränti R. Mariescu-Istodor and C. Zhong, &quot;XNN graph&quot;</span>
<span class="sd">    IAPR Joint Int. Workshop on Structural, Syntactic, and Statistical Pattern</span>
<span class="sd">    Recognition Merida, Mexico, LNCS 10029, 207-217, November 2016.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="n">urls</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/g2-txt.zip&quot;</span><span class="p">]</span>

<div class="viewcode-block" id="G2.prepare"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.G2.prepare">[docs]</a>    <span class="k">def</span> <span class="nf">prepare</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dim</span><span class="p">,</span> <span class="n">sd</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Make torch Tensors from g2-`dim`-`sd` and infer labels.</span>
<span class="sd">        Args:</span>
<span class="sd">            dim:</span>
<span class="sd">            sd:</span>

<span class="sd">        Returns:</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">filename</span> <span class="o">=</span> <span class="s1">&#39;g2-</span><span class="si">{}</span><span class="s1">-</span><span class="si">{}</span><span class="s1">.txt&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">dim</span><span class="p">,</span> <span class="n">sd</span><span class="p">)</span>
        <span class="n">data</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="n">target</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">root</span><span class="p">,</span> <span class="n">filename</span><span class="p">))</span> <span class="k">as</span> <span class="n">in_f</span><span class="p">:</span>
            <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">line</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">in_f</span><span class="p">):</span>
                <span class="n">a</span><span class="p">,</span> <span class="n">b</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="nb">int</span><span class="p">,</span> <span class="n">line</span><span class="o">.</span><span class="n">split</span><span class="p">())),</span> <span class="mi">0</span> <span class="k">if</span> <span class="n">i</span> <span class="o">&lt;</span> <span class="mi">1024</span> <span class="k">else</span> <span class="mi">1</span>
                <span class="n">data</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">a</span><span class="p">)</span>
                <span class="n">target</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">b</span><span class="p">)</span>
        <span class="n">data</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
        <span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">(</span><span class="n">target</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">stardardize</span><span class="p">:</span>
            <span class="n">data</span> <span class="o">=</span> <span class="p">(</span><span class="n">data</span> <span class="o">-</span> <span class="mi">550</span><span class="p">)</span> <span class="o">/</span> <span class="mi">50</span>

        <span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">target</span></div>

<div class="viewcode-block" id="G2.check_exists"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.G2.check_exists">[docs]</a>    <span class="k">def</span> <span class="nf">check_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>

<span class="sd">        Returns:</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">root</span><span class="p">,</span> <span class="s1">&#39;g2-1-10.txt&#39;</span><span class="p">))</span></div></div>


<div class="viewcode-block" id="S_set"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.S_set">[docs]</a><span class="k">class</span> <span class="nc">S_set</span><span class="p">(</span><span class="n">_SmallDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Download and use S-sets dataset.</span>

<span class="sd">    Synthetic 2-d data with N=5000 vectors and k=15 Gaussian clusters</span>
<span class="sd">    with different degree of cluster overlapping.</span>

<span class="sd">    Select Arguments</span>
<span class="sd">    ----------------</span>
<span class="sd">    num : int</span>
<span class="sd">       Higher `num` means, higher chance of overlapping between the modes.</span>
<span class="sd">       Choose: [1, 2, 3, 4]</span>

<span class="sd">    P. Fränti and O. Virmajoki,</span>
<span class="sd">    &quot;Iterative shrinking method for clustering problems&quot;,</span>
<span class="sd">    Pattern Recognition, 39 (5), 761-765, May 2006.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="n">urls</span> <span class="o">=</span> <span class="p">[</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/s1.txt&quot;</span><span class="p">,</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/s2.txt&quot;</span><span class="p">,</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/s3.txt&quot;</span><span class="p">,</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/s4.txt&quot;</span><span class="p">,</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/s-originals.zip&quot;</span>
    <span class="p">]</span>

    <span class="n">sync_files</span> <span class="o">=</span> <span class="mi">5</span>

<div class="viewcode-block" id="S_set.files"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.S_set.files">[docs]</a>    <span class="k">def</span> <span class="nf">files</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">num</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Make torch Tensors from &#39;s{num}.txt&#39; and fetch labels.</span>
<span class="sd">        Args:</span>
<span class="sd">            num:</span>

<span class="sd">        Returns:</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="s1">&#39;s</span><span class="si">{}</span><span class="s1">.txt&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">num</span><span class="p">),</span> <span class="s1">&#39;s</span><span class="si">{}</span><span class="s1">-label.pa&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">num</span><span class="p">)</span></div>

<div class="viewcode-block" id="S_set.check_exists"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.S_set.check_exists">[docs]</a>    <span class="k">def</span> <span class="nf">check_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">root</span><span class="p">,</span> <span class="s1">&#39;s1.txt&#39;</span><span class="p">))</span></div></div>


<div class="viewcode-block" id="A_set"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.A_set">[docs]</a><span class="k">class</span> <span class="nc">A_set</span><span class="p">(</span><span class="n">_SmallDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Download and use A-sets dataset.</span>

<span class="sd">    Synthetic 2-d data with varying number of vectors (N) and clusters (k).</span>
<span class="sd">    There are 150 vectors per cluster.</span>

<span class="sd">    Select Arguments</span>
<span class="sd">    ----------------</span>
<span class="sd">    num : int</span>
<span class="sd">       Higher `num` means, higher chance of overlapping between the modes.</span>
<span class="sd">       Choose: [1, 2, 3]</span>

<span class="sd">    A1: N=3000, k=20</span>
<span class="sd">    A2: N=5250, k=35</span>
<span class="sd">    A3: N=7500, k=50</span>

<span class="sd">    I. Kärkkäinen and P. Fränti,</span>
<span class="sd">    &quot;Dynamic local search algorithm for the clustering problem&quot;,</span>
<span class="sd">    Research Report A-2002-6</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="n">urls</span> <span class="o">=</span> <span class="p">[</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/a1.txt&quot;</span><span class="p">,</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/a2.txt&quot;</span><span class="p">,</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/a3.txt&quot;</span><span class="p">,</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/a-gt-pa.zip&quot;</span>
    <span class="p">]</span>

    <span class="n">sync_files</span> <span class="o">=</span> <span class="mi">4</span>

<div class="viewcode-block" id="A_set.files"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.A_set.files">[docs]</a>    <span class="k">def</span> <span class="nf">files</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">num</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>

<span class="sd">        Args:</span>
<span class="sd">            num:</span>

<span class="sd">        Returns:</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="s1">&#39;a</span><span class="si">{}</span><span class="s1">.txt&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">num</span><span class="p">),</span> <span class="s1">&#39;a</span><span class="si">{}</span><span class="s1">-ga.pa&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">num</span><span class="p">)</span></div>

<div class="viewcode-block" id="A_set.check_exists"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.A_set.check_exists">[docs]</a>    <span class="k">def</span> <span class="nf">check_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>

<span class="sd">        Returns:</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">root</span><span class="p">,</span> <span class="s1">&#39;a1.txt&#39;</span><span class="p">))</span></div></div>


<div class="viewcode-block" id="DIM_set"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.DIM_set">[docs]</a><span class="k">class</span> <span class="nc">DIM_set</span><span class="p">(</span><span class="n">_SmallDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Download and use the (high) DIM-sets dataset.</span>

<span class="sd">    High-dimensional data sets N=1024 and k=16 Gaussian clusters.</span>

<span class="sd">    Select arguments</span>
<span class="sd">    ----------------</span>
<span class="sd">    dim : int</span>
<span class="sd">       Dimension of the input space. Choose: [32, 64, 128, 256, 512, 1024]</span>

<span class="sd">    P. Fränti, O. Virmajoki and V. Hautamäki,</span>
<span class="sd">    &quot;Fast agglomerative clustering using a k-nearest neighbor graph&quot;,</span>
<span class="sd">    IEEE Trans. on Pattern Analysis and Machine Intelligence, 28 (11),</span>
<span class="sd">    1875-1881, November 2006.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="n">urls</span> <span class="o">=</span> <span class="p">[</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/dim032.txt&quot;</span><span class="p">,</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/dim032.pa&quot;</span><span class="p">,</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/dim064.txt&quot;</span><span class="p">,</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/dim064.pa&quot;</span><span class="p">,</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/dim128.txt&quot;</span><span class="p">,</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/dim128.pa&quot;</span><span class="p">,</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/dim256.txt&quot;</span><span class="p">,</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/dim256.pa&quot;</span><span class="p">,</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/dim512.txt&quot;</span><span class="p">,</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/dim512.pa&quot;</span><span class="p">,</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/dim1024.txt&quot;</span><span class="p">,</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/dim1024.pa&quot;</span><span class="p">,</span>
    <span class="p">]</span>

    <span class="n">sync_files</span> <span class="o">=</span> <span class="mi">5</span>

<div class="viewcode-block" id="DIM_set.files"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.DIM_set.files">[docs]</a>    <span class="k">def</span> <span class="nf">files</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dim</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>

<span class="sd">        Args:</span>
<span class="sd">            dim:</span>

<span class="sd">        Returns:</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="s1">&#39;dim</span><span class="si">{:03d}</span><span class="s1">.txt&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">dim</span><span class="p">),</span> <span class="s1">&#39;dim</span><span class="si">{:03d}</span><span class="s1">.pa&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">dim</span><span class="p">)</span></div>

<div class="viewcode-block" id="DIM_set.check_exists"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.DIM_set.check_exists">[docs]</a>    <span class="k">def</span> <span class="nf">check_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>

<span class="sd">        Returns:</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">root</span><span class="p">,</span> <span class="s1">&#39;dim032.txt&#39;</span><span class="p">))</span></div></div>


<div class="viewcode-block" id="Unbalance"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.Unbalance">[docs]</a><span class="k">class</span> <span class="nc">Unbalance</span><span class="p">(</span><span class="n">_SmallDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Download and use the Unbalance dataset.</span>

<span class="sd">    Synthetic 2-d data with N=6500 vectors and k=8 Gaussian clusters</span>

<span class="sd">    There are 3 &quot;dense&quot; clusters of 2000 vectors each and</span>
<span class="sd">    5 &quot;sparse&quot; clusters of 100 vectors each.</span>

<span class="sd">    M. Rezaei and P. Fränti,</span>
<span class="sd">    &quot;Set-matching methods for external cluster validity&quot;,</span>
<span class="sd">    IEEE Trans. on Knowledge and Data Engineering, 28 (8), 2173-2186,</span>
<span class="sd">    August 2016.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="n">urls</span> <span class="o">=</span> <span class="p">[</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/unbalance.txt&quot;</span><span class="p">,</span>
        <span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/unbalance-gt-pa.zip&quot;</span><span class="p">,</span>
    <span class="p">]</span>

    <span class="n">sync_files</span> <span class="o">=</span> <span class="mi">4</span>

<div class="viewcode-block" id="Unbalance.files"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.Unbalance.files">[docs]</a>    <span class="k">def</span> <span class="nf">files</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>

<span class="sd">        Returns:</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="s1">&#39;unbalance.txt&#39;</span><span class="p">,</span> <span class="s1">&#39;unbalance-gt.pa&#39;</span></div>

<div class="viewcode-block" id="Unbalance.check_exists"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.Unbalance.check_exists">[docs]</a>    <span class="k">def</span> <span class="nf">check_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>

<span class="sd">        Returns:</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">data</span><span class="p">,</span> <span class="n">labels</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">()</span>
        <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">root</span><span class="p">,</span> <span class="n">data</span><span class="p">))</span> <span class="ow">and</span>\
            <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">root</span><span class="p">,</span> <span class="n">labels</span><span class="p">))</span></div></div>


<span class="k">class</span> <span class="nc">_Shapes</span><span class="p">(</span><span class="n">_SmallDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Wrap shapes datasets from the website.&quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="nf">prepare</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Make torch Tensors from data and label files.</span>
<span class="sd">        Returns:</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">datafile</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">urls</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">rpartition</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)[</span><span class="mi">2</span><span class="p">]</span>
        <span class="n">data_filepath</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">root</span><span class="p">,</span> <span class="n">datafile</span><span class="p">)</span>
        <span class="n">data</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="n">target</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">data_filepath</span><span class="p">)</span> <span class="k">as</span> <span class="n">data_f</span><span class="p">:</span>
            <span class="k">for</span> <span class="n">sample</span> <span class="ow">in</span> <span class="n">data_f</span><span class="p">:</span>
                <span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">label</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="nb">float</span><span class="p">,</span> <span class="n">sample</span><span class="o">.</span><span class="n">split</span><span class="p">()))</span>
                <span class="n">data</span><span class="o">.</span><span class="n">append</span><span class="p">([</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">])</span>
                <span class="n">target</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">label</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span>
        <span class="n">data</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
        <span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">(</span><span class="n">target</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">stardardize</span><span class="p">:</span>
            <span class="n">data_mean</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">keepdim</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
            <span class="n">data_std</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">std</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">keepdim</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
            <span class="n">data</span> <span class="o">=</span> <span class="p">(</span><span class="n">data</span> <span class="o">-</span> <span class="n">data_mean</span><span class="p">)</span> <span class="o">/</span> <span class="n">data_std</span>

        <span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">target</span>

    <span class="k">def</span> <span class="nf">check_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>

<span class="sd">        Returns:</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">datafile</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">urls</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">rpartition</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)[</span><span class="mi">2</span><span class="p">]</span>
        <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">root</span><span class="p">,</span> <span class="n">datafile</span><span class="p">))</span>


<div class="viewcode-block" id="Aggregation"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.Aggregation">[docs]</a><span class="k">class</span> <span class="nc">Aggregation</span><span class="p">(</span><span class="n">_Shapes</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Download and use the Aggregation dataset.</span>

<span class="sd">    N=788, k=7, D=2</span>

<span class="sd">    A. Gionis, H. Mannila, and P. Tsaparas,</span>
<span class="sd">    Clustering aggregation.</span>
<span class="sd">    ACM Transactions on Knowledge Discovery from Data (TKDD), 2007.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="n">urls</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/Aggregation.txt&quot;</span><span class="p">]</span></div>


<div class="viewcode-block" id="Compound"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.Compound">[docs]</a><span class="k">class</span> <span class="nc">Compound</span><span class="p">(</span><span class="n">_Shapes</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Download and use the Compound dataset.</span>

<span class="sd">    N=399, k=6, D=2</span>

<span class="sd">    C.T. Zahn,</span>
<span class="sd">    Graph-theoretical methods for detecting and describing gestalt clusters.</span>
<span class="sd">    IEEE Transactions on Computers, 1971.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="n">urls</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/Compound.txt&quot;</span><span class="p">]</span></div>


<div class="viewcode-block" id="Pathbased"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.Pathbased">[docs]</a><span class="k">class</span> <span class="nc">Pathbased</span><span class="p">(</span><span class="n">_Shapes</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Download and use the Pathbased dataset.</span>

<span class="sd">    N=300, k=3, D=2</span>

<span class="sd">    H. Chang and D.Y. Yeung,</span>
<span class="sd">    Robust path-based spectral clustering.</span>
<span class="sd">    Pattern Recognition, 2008.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="n">urls</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/pathbased.txt&quot;</span><span class="p">]</span></div>


<div class="viewcode-block" id="Spiral"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.Spiral">[docs]</a><span class="k">class</span> <span class="nc">Spiral</span><span class="p">(</span><span class="n">_Shapes</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Download and use the Spiral dataset.</span>

<span class="sd">    N=312, k=3, D=2</span>

<span class="sd">    H. Chang and D.Y. Yeung,</span>
<span class="sd">    Robust path-based spectral clustering.</span>
<span class="sd">    Pattern Recognition, 2008.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="n">urls</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/spiral.txt&quot;</span><span class="p">]</span></div>


<div class="viewcode-block" id="D31"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.D31">[docs]</a><span class="k">class</span> <span class="nc">D31</span><span class="p">(</span><span class="n">_Shapes</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Download and use the D31 dataset.</span>

<span class="sd">    N=3100, k=31, D=2</span>

<span class="sd">    C.J. Veenman, M.J.T. Reinders, and E. Backer,</span>
<span class="sd">    A maximum variance cluster algorithm.</span>
<span class="sd">    IEEE Trans. Pattern Analysis and Machine Intelligence 2002.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="n">urls</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/D31.txt&quot;</span><span class="p">]</span></div>


<div class="viewcode-block" id="R15"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.R15">[docs]</a><span class="k">class</span> <span class="nc">R15</span><span class="p">(</span><span class="n">_Shapes</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Download and use the R15 dataset.</span>

<span class="sd">    N=600, k=15, D=2</span>

<span class="sd">    C.J. Veenman, M.J.T. Reinders, and E. Backer,</span>
<span class="sd">    A maximum variance cluster algorithm.</span>
<span class="sd">    IEEE Trans. Pattern Analysis and Machine Intelligence 2002.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="n">urls</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/R15.txt&quot;</span><span class="p">]</span></div>


<div class="viewcode-block" id="Jain"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.Jain">[docs]</a><span class="k">class</span> <span class="nc">Jain</span><span class="p">(</span><span class="n">_Shapes</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Download and use the ORIGINAL (2 moons) Jain dataset.</span>

<span class="sd">    N=373, k=2, D=2</span>

<span class="sd">    A. Jain and M. Law,</span>
<span class="sd">    Data clustering: A user&#39;s dilemma.</span>
<span class="sd">    Lecture Notes in Computer Science, 2005.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="n">urls</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/jain.txt&quot;</span><span class="p">]</span></div>


<div class="viewcode-block" id="Flame"><a class="viewcode-back" href="../../../../cortex.built_ins.datasets.html#cortex.built_ins.datasets.toysets.Flame">[docs]</a><span class="k">class</span> <span class="nc">Flame</span><span class="p">(</span><span class="n">_Shapes</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Download and use the Flame dataset.</span>

<span class="sd">    N=240, k=2, D=2</span>

<span class="sd">    L. Fu and E. Medico,</span>
<span class="sd">    FLAME, a novel fuzzy clustering method for the analysis of DNA microarray</span>
<span class="sd">    data. BMC bioinformatics, 2007.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="n">urls</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;http://cs.joensuu.fi/sipu/datasets/flame.txt&quot;</span><span class="p">]</span></div>
</pre></div>

           </div>
           
          </div>
          <footer>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2018, MILA.

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  


  

    <script type="text/javascript">
        var DOCUMENTATION_OPTIONS = {
            URL_ROOT:'../../../../',
            VERSION:'',
            LANGUAGE:'None',
            COLLAPSE_INDEX:false,
            FILE_SUFFIX:'.html',
            HAS_SOURCE:  true,
            SOURCELINK_SUFFIX: '.txt'
        };
    </script>
      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
      <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>

  

  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script> 

</body>
</html>